{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Analyze how the batch size affects the performance across installed Caffe variants and models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**NB:** This is an early version of this notebook. Please see e.g. http://github.com/dividiti/ck-caffe-nvidia-tx1 for a more robust and up-to-date example."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Includes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Standard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import json\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Scientific"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import IPython as ip\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "import matplotlib as mp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print ('IPython version: %s' % ip.__version__)\n",
    "print ('NumPy version: %s' % np.__version__)\n",
    "print ('SciPy version: %s' % sp.__version__)\n",
    "print ('Pandas version: %s' % pd.__version__)\n",
    "print ('Matplotlib version: %s' % mp.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "if mp.__version__[0]=='2': mp.style.use('classic')\n",
    "from matplotlib import cm\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import display\n",
    "def display_in_full(df):\n",
    "    pd.options.display.max_columns = len(df.columns)\n",
    "    pd.options.display.max_rows = len(df.index)\n",
    "    display(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Collective Knowledge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import ck.kernel as ck\n",
    "print ('CK version: %s' % ck.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Access Caffe experimental data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_experimental_results(tags='explore-batch-size-libs-models', repo_uoa='local'):\n",
    "    # Get (lib_tag, model_tag) from a list of tags that should be available in r['dict']['tags'].\n",
    "    # Tags include 2 of the 3 irrelevant tags (e.g. command name), a model tag and lib tag.\n",
    "    # NB: Since it's easier to list a few model tags than many lib tags, the latter list is not expicitly specified.\n",
    "    def get_lib_model_tags(tags):\n",
    "        irrelevant_tags = [ 'explore-batch-size-libs-models', 'time_gpu', 'time_cpu', 'default' ]\n",
    "        model_tags = [ 'bvlc-alexnet', 'bvlc-googlenet', 'deepscale-squeezenet-1.0', 'deepscale-squeezenet-1.1' ]\n",
    "        lib_model_tags = [ tag for tag in tags if tag not in irrelevant_tags ]\n",
    "        model_tags = [ tag for tag in lib_model_tags if tag in model_tags ]\n",
    "        lib_tags = [ tag for tag in lib_model_tags if tag not in model_tags ]\n",
    "        return (lib_tags[0], model_tags[0])\n",
    "    \n",
    "    module_uoa = 'experiment'\n",
    "    r = ck.access({'action':'search', 'repo_uoa':repo_uoa, 'module_uoa':module_uoa, 'tags':tags})\n",
    "    if r['return']>0:\n",
    "        print (\"Error: %s\" % r['error'])\n",
    "        exit(1)\n",
    "    experiments = r['lst']\n",
    "    \n",
    "    first_experiment = True\n",
    "    for experiment in experiments:\n",
    "        data_uoa = experiment['data_uoa']\n",
    "        r = ck.access({'action':'list_points', 'repo_uoa':repo_uoa, 'module_uoa':module_uoa, 'data_uoa':data_uoa})\n",
    "        if r['return']>0:\n",
    "            print (\"Error: %s\" % r['error'])\n",
    "            exit(1)\n",
    "       \n",
    "        results = []\n",
    "        for point in r['points']:\n",
    "            with open(os.path.join(r['path'], 'ckp-%s.0001.json' % point)) as point_file:\n",
    "                point_data_raw = json.load(point_file)\n",
    "                run_info_list = [\n",
    "                    characteristics['run']\n",
    "                    for characteristics in point_data_raw['characteristics_list']\n",
    "                    if characteristics['run'].get('run_success','')!=''\n",
    "                ]\n",
    "                # Select characteristics of interest. TODO: simplify.\n",
    "                point_data_dict = {\n",
    "                    'lib'         : get_lib_model_tags(r['dict']['tags'])[0],\n",
    "                    'model'       : get_lib_model_tags(r['dict']['tags'])[1],\n",
    "                    'batch size'  : int(point_data_raw['choices']['env']['CK_CAFFE_BATCH_SIZE']),\n",
    "                    'time (ms)'   : [ float(run_info.get('time_fw_ms',0)) for run_info in run_info_list ],\n",
    "                    'memory (MB)' : [ int(run_info.get('memory_mbytes',0)) for run_info in run_info_list ],\n",
    "                    'success?'    : [ run_info.get('run_success','n/a') for run_info in run_info_list ]\n",
    "                }\n",
    "                results.append(point_data_dict)\n",
    "        df_new = pd.DataFrame(data=results)\n",
    "        df_new = df_new.set_index(['lib', 'model', 'batch size'])\n",
    "        # Need to convert lists into separate columns. Ugly but works.\n",
    "        # NB: More beautiful code can be found e.g. at http://github.com/dividiti/ck-caffe-nvidia-tx1.\n",
    "        df_new_memory = df_new['memory (MB)'].apply(pd.Series)\n",
    "        df_new_memory.columns = [ ['memory (MB)']*len(df_new_memory.columns), df_new_memory.columns ]\n",
    "        df_new_time = df_new['time (ms)'].apply(pd.Series)\n",
    "        df_new_time.columns = [ ['time (ms)']*len(df_new_time.columns), df_new_time.columns ]\n",
    "        df_new_success = df_new['success?'].apply(pd.Series)\n",
    "        df_new_success.columns = [ ['success?']*len(df_new_success.columns), df_new_success.columns ]               \n",
    "        # Join together.\n",
    "        df_new = df_new_memory.join(df_new_time).join(df_new_success)\n",
    "        df_new.columns.names = ['characteristic', 'repetition']\n",
    "        df_new = df_new.stack('repetition').unstack(['lib', 'model'])\n",
    "        # display_in_full(df_new)\n",
    "        if first_experiment:\n",
    "            first_experiment = False\n",
    "            df_all = df_new\n",
    "        else:\n",
    "            df_all = df_all.join(df_new)\n",
    "    return df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_all = get_experimental_results(repo_uoa='ck-caffe-odroid-xu3-thresh')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## All execution time data indexed by repetitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_time = df_all['time (ms)'].unstack('batch size').apply(pd.to_numeric).sortlevel(level=['lib', 'model'], axis=1)\n",
    "display_in_full(df_time)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Mean execution time per batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_mean_time_per_batch = df_time.describe().ix['mean'].unstack(level='batch size')\n",
    "display_in_full(df_mean_time_per_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "batch_sizes = df_mean_time_per_batch.columns.tolist()\n",
    "batch_sizes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Mean execution time per image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_mean_time_per_image = df_mean_time_per_batch / batch_sizes\n",
    "display_in_full(df_mean_time_per_image)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Best mean execution time per image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_mean_time_per_image.min(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# What is the batch size that gives the minimum time per image (or the maximum number of images per second)?\n",
    "df_mean_time_per_image.idxmin(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Use the batch size with the best mean execution time per image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_time_per_image = df_time / (batch_sizes*(len(df_time.columns)/len(batch_sizes)))\n",
    "display_in_full(df_time_per_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_min_time_per_image_index = pd.DataFrame(df_mean_time_per_image.idxmin(axis=1)).set_index(0, append=True).index.values\n",
    "df_model_lib = df_time_per_image[df_min_time_per_image_index] \\\n",
    "     .stack(['model', 'lib']).reorder_levels(['model','lib','repetition'])\n",
    "df_model_lib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_model_lib_mean = df_model_lib.groupby(level=['model', 'lib']).mean()\n",
    "df_model_lib_std  = df_model_lib.groupby(level=['model', 'lib']).std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot(mean, std, ymax=0, title='Execution time per image (ms)'):\n",
    "    ymax = mean.max().max() if ymax==0 else ymax\n",
    "    mean.plot(yerr=std, title=title, kind='bar', ylim=[0,ymax*1.05],  rot=0, figsize=[16, 8], grid=True, legend=True, colormap=cm.autumn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Plot by Caffe models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### All"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "mean = df_model_lib_mean.unstack('lib')\n",
    "std  = df_model_lib_std.unstack('lib')\n",
    "plot(mean, std)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Selection: AlexNet, SqueezeNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_model_lib_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "mean = df_model_lib_mean.ix[['bvlc-alexnet', 'deepscale-squeezenet-1.1']].unstack('lib')\n",
    "std  = df_model_lib_std.ix[['bvlc-alexnet', 'deepscale-squeezenet-1.1']].unstack('lib')\n",
    "plot(mean, std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_model_lib_mean.ix[['bvlc-alexnet', 'deepscale-squeezenet-1.1']].unstack('lib').iloc[1] / \\\n",
    "df_model_lib_mean.ix[['bvlc-alexnet', 'deepscale-squeezenet-1.1']].unstack('lib').iloc[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Selection: CUDA-level performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Plot by Caffe libs (variants)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### All"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "mean = df_model_lib_mean.unstack('model')\n",
    "std  = df_model_lib_std.unstack('model')\n",
    "plot(mean, std)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Selection: AlexNet-level accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "alexnet_level_accuracy = [ 'bvlc-alexnet', 'deepscale-squeezenet-1.1' ]\n",
    "mean = df_model_lib_mean.ix[alexnet_level_accuracy].unstack('model')\n",
    "std  = df_model_lib_std.ix[alexnet_level_accuracy].unstack('model')\n",
    "plot(mean, std)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Selection: AlexNet-level accuracy, CUDA-level performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# mean = mean.ix[cuda_level_performance]\n",
    "# std = std.ix[cuda_level_performance]\n",
    "# plot(mean, std)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## All memory size data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Batch size of 2; repetition 0 (should be always available).\n",
    "df_memory = df_all['memory (MB)'].loc[2].loc[0].unstack('lib')\n",
    "plot(df_memory, pd.DataFrame(), title='Memory consumption (MB)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_memory.ix[['bvlc-alexnet', 'deepscale-squeezenet-1.1']].iloc[1] / \\\n",
    "df_memory.ix[['bvlc-alexnet', 'deepscale-squeezenet-1.1']].iloc[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
